# Copyright (c) 2025 Huawei Technologies Co., Ltd.
#
# openMind is licensed under Mulan PSL v2.
# You can use this software according to the terms and conditions of the Mulan PSL v2.
# You may obtain a copy of Mulan PSL v2 at:
#
#          http://license.coscl.org.cn/MulanPSL2
#
# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
# See the Mulan PSL v2 for more details.


import sys
from openmind.flow.arguments import initialize_openmind, get_args
from openmind.utils import logging
from openmind.flow.eval.lmeval import LMEval

logger = logging.get_logger(name=__name__, allow_line_separator=True)
logging.set_verbosity_info()

LMEVAL = "lmeval"
SUPPORT_BACKEND = (LMEVAL,)


def run_eval(**kwargs):
    """Evaluate LLM with EleutherAI's llm evaluation harness."""

    command_line = sys.argv[1::]
    if len(command_line) == 2 and command_line[-1].endswith("yaml"):
        yaml_file = command_line[-1]
    else:
        yaml_file = None

    initialize_openmind(yaml_file, **kwargs)

    args = get_args()

    if not args.backend:
        LMEval().evaluate(args)
    elif args.backend in SUPPORT_BACKEND:
        if args.backend == LMEVAL:
            LMEval().evaluate(args)
    else:
        logger.error(f"Eval interface currently supports {SUPPORT_BACKEND} as backend")
